{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "99659e8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import sys \n",
    "sys.path.append(\"..\") \n",
    "sys.path.append(\"../ma-gym\")\n",
    "import rl_utils\n",
    "from ma_gym.envs.combat.combat import Combat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5cc4cba0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PolicyNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim):\n",
    "        super(PolicyNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.fc3 = torch.nn.Linear(hidden_dim, action_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc2(F.relu(self.fc1(x))))\n",
    "        return F.softmax(self.fc3(x), dim=1)\n",
    "\n",
    "\n",
    "class ValueNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim):\n",
    "        super(ValueNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.fc3 = torch.nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc2(F.relu(self.fc1(x))))\n",
    "        return self.fc3(x)\n",
    "\n",
    "\n",
    "class PPO:\n",
    "    ''' PPO算法,采用截断方式 '''\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim, actor_lr, critic_lr,\n",
    "                 lmbda, eps, gamma, device):\n",
    "        self.actor = PolicyNet(state_dim, hidden_dim, action_dim).to(device)\n",
    "        self.critic = ValueNet(state_dim, hidden_dim).to(device)\n",
    "        self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),\n",
    "                                                lr=actor_lr)\n",
    "        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),\n",
    "                                                 lr=critic_lr)\n",
    "        self.gamma = gamma\n",
    "        self.lmbda = lmbda\n",
    "        self.eps = eps  # PPO中截断范围的参数\n",
    "        self.device = device\n",
    "\n",
    "    def take_action(self, state):\n",
    "        state = torch.tensor([state], dtype=torch.float).to(self.device)\n",
    "        probs = self.actor(state)\n",
    "        action_dist = torch.distributions.Categorical(probs)\n",
    "        action = action_dist.sample()\n",
    "        return action.item()\n",
    "\n",
    "    def update(self, transition_dict):\n",
    "        states = torch.tensor(transition_dict['states'],\n",
    "                              dtype=torch.float).to(self.device)\n",
    "        actions = torch.tensor(transition_dict['actions']).view(-1, 1).to(\n",
    "            self.device)\n",
    "        rewards = torch.tensor(transition_dict['rewards'],\n",
    "                               dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        next_states = torch.tensor(transition_dict['next_states'],\n",
    "                                   dtype=torch.float).to(self.device)\n",
    "        dones = torch.tensor(transition_dict['dones'],\n",
    "                             dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        td_target = rewards + self.gamma * self.critic(next_states) * (1 -\n",
    "                                                                       dones)\n",
    "        td_delta = td_target - self.critic(states)\n",
    "        advantage = rl_utils.compute_advantage(self.gamma, self.lmbda,\n",
    "                                               td_delta.cpu()).to(self.device)\n",
    "        old_log_probs = torch.log(self.actor(states).gather(1,\n",
    "                                                            actions)).detach()\n",
    "\n",
    "        log_probs = torch.log(self.actor(states).gather(1, actions))\n",
    "        ratio = torch.exp(log_probs - old_log_probs)\n",
    "        surr1 = ratio * advantage\n",
    "        surr2 = torch.clamp(ratio, 1 - self.eps,\n",
    "                            1 + self.eps) * advantage  # 截断\n",
    "        actor_loss = torch.mean(-torch.min(surr1, surr2))  # PPO损失函数\n",
    "        critic_loss = torch.mean(\n",
    "            F.mse_loss(self.critic(states), td_target.detach()))\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        actor_loss.backward()\n",
    "        critic_loss.backward()\n",
    "        self.actor_optimizer.step()\n",
    "        self.critic_optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ef805bb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/d/Code/Python/RL-Learning/.venv/lib/python3.11/site-packages/gym/spaces/box.py:127: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  logger.warn(f\"Box bound precision lowered by casting to {self.dtype}\")\n",
      "Iter 0: 100%|██████████| 100/100 [00:04<00:00, 24.72it/s, episode=100, return=0.000]\n",
      "Iter 1: 100%|██████████| 100/100 [00:04<00:00, 24.13it/s, episode=200, return=0.000]\n",
      "Iter 2: 100%|██████████| 100/100 [00:04<00:00, 24.59it/s, episode=300, return=0.000]\n",
      "Iter 3: 100%|██████████| 100/100 [00:03<00:00, 28.04it/s, episode=400, return=0.000]\n",
      "Iter 4:  69%|██████▉   | 69/100 [00:03<00:01, 16.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 4:  94%|█████████▍| 94/100 [00:04<00:00, 22.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 4: 100%|██████████| 100/100 [00:04<00:00, 20.97it/s, episode=500, return=0.020]\n",
      "Iter 5:  27%|██▋       | 27/100 [00:01<00:03, 20.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 5:  65%|██████▌   | 65/100 [00:03<00:01, 19.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 5: 100%|██████████| 100/100 [00:04<00:00, 20.51it/s, episode=600, return=0.020]\n",
      "Iter 6:  28%|██▊       | 28/100 [00:01<00:02, 31.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 6:  55%|█████▌    | 55/100 [00:02<00:01, 25.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 6:  91%|█████████ | 91/100 [00:04<00:00, 16.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 6: 100%|██████████| 100/100 [00:04<00:00, 20.28it/s, episode=700, return=0.030]\n",
      "Iter 7:  33%|███▎      | 33/100 [00:01<00:02, 25.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 7:  55%|█████▌    | 55/100 [00:02<00:02, 18.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 7:  68%|██████▊   | 68/100 [00:03<00:01, 20.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 7: 100%|██████████| 100/100 [00:05<00:00, 19.76it/s, episode=800, return=0.040]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 8:   9%|▉         | 9/100 [00:00<00:04, 21.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 8:  73%|███████▎  | 73/100 [00:03<00:01, 20.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 8:  90%|█████████ | 90/100 [00:04<00:00, 33.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 8: 100%|██████████| 100/100 [00:04<00:00, 21.59it/s, episode=900, return=0.030]\n",
      "Iter 9:  44%|████▍     | 44/100 [00:01<00:02, 21.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n",
      "win_list [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iter 9: 100%|██████████| 100/100 [00:04<00:00, 21.80it/s, episode=1000, return=0.020]\n"
     ]
    }
   ],
   "source": [
    "actor_lr = 3e-4\n",
    "critic_lr = 1e-3\n",
    "# num_episodes = 100000\n",
    "num_episodes = 1000\n",
    "hidden_dim = 64\n",
    "gamma = 0.99\n",
    "lmbda = 0.97\n",
    "eps = 0.2\n",
    "device = 'cpu'\n",
    "\n",
    "team_size = 2\n",
    "grid_size = (15, 15)\n",
    "# 创建Combat环境，格子世界的大小为15x15，己方智能体和敌方智能体数量都为2\n",
    "env = Combat(grid_shape=grid_size, n_agents=team_size, n_opponents=team_size)\n",
    "state_dim = env.observation_space[0].shape[0]\n",
    "action_dim = env.action_space[0].n\n",
    "\n",
    "#两个智能体共享同一个策略\n",
    "agent = PPO(state_dim, hidden_dim, action_dim, actor_lr, critic_lr, lmbda, eps, gamma, device)\n",
    "\n",
    "win_list = []\n",
    "for i in range(10):\n",
    "    with tqdm(total=int(num_episodes/10), desc='Iter %d' % i) as pbar:\n",
    "        for i_episode in range(int(num_episodes / 10)):\n",
    "            trainsition_dict_1 = {\n",
    "                'states':[], 'actions':[], 'next_states':[], 'rewards':[], 'dones':[]\n",
    "            }\n",
    "            trainsition_dict_2 = {\n",
    "                'states':[], 'actions':[], 'next_states':[], 'rewards':[], 'dones':[]\n",
    "            }\n",
    "            s = env.reset()\n",
    "            terminal = False\n",
    "            game_step = 0\n",
    "            while not terminal:\n",
    "                game_step += 1\n",
    "                a_1 = agent.take_action(s[0])\n",
    "                a_2 = agent.take_action(s[1])\n",
    "                next_s, r, done, info = env.step([a_1, a_2])\n",
    "                '''\n",
    "                next_s.shape torch.Size([2, 150])\n",
    "                r.shape torch.Size([2])\n",
    "                done.shape torch.Size([2])\n",
    "                info.shape {'health': {0: 3, 1: 3}, 'win': False}\n",
    "                '''\n",
    "                trainsition_dict_1['states'].append(s[0])\n",
    "                trainsition_dict_1['actions'].append(a_1)\n",
    "                trainsition_dict_1['next_states'].append(next_s[0])\n",
    "                # 奖励设计：如果赢了+100，否则-0.1\n",
    "                trainsition_dict_1['rewards'].append(r[0] + 100 if info['win'] else r[0] - 0.1)\n",
    "                trainsition_dict_1['dones'].append(False)\n",
    "\n",
    "                trainsition_dict_2['states'].append(s[1])\n",
    "                trainsition_dict_2['actions'].append(a_2)\n",
    "                trainsition_dict_2['next_states'].append(next_s[1])\n",
    "                trainsition_dict_2['rewards'].append(r[1] + 100 if info['win'] else r[1] - 0.1)\n",
    "                trainsition_dict_2['dones'].append(False)\n",
    "            \n",
    "                s = next_s\n",
    "                terminal = all(done)\n",
    "\n",
    "                # if terminal :\n",
    "                #     game_win = '赢了' if info['win'] else '输了'\n",
    "                #     print(f'这局游戏走了{game_step}步, {game_win}!')\n",
    "\n",
    "            win_list.append(1 if info['win'] else 0)\n",
    "            if info['win']:\n",
    "                print('win_list', win_list)\n",
    "            agent.update(trainsition_dict_1)\n",
    "            agent.update(trainsition_dict_2)\n",
    "            if (i_episode + 1) % 100 == 0:\n",
    "                pbar.set_postfix(\n",
    "                    {\n",
    "                        'episode': '%d' % (num_episodes / 10 * i + i_episode + 1),\n",
    "                        'return': '%.3f' % np.mean(win_list[-100:])\n",
    "                    }\n",
    "                )\n",
    "            pbar.update(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "48cbc269",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAHHCAYAAACfqw0dAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAZ/dJREFUeJzt3XtcVHX+P/DXDDAzXOQuN0XBK5qmhooYeQFWKnZbdmsr19L8WlppWXQxy1u1hWmWXVxdd0vbX7m67JqZmpuCZiqionhL8IqoOFxEZhDkNvP5/YFzZOIi4AxnGF7Px2MeyjmfOec9M8q8P+fzOe+PQgghQEREREQtopQ7ACIiIqL2iEkUERERUSswiSIiIiJqBSZRRERERK3AJIqIiIioFZhEEREREbUCkygiIiKiVmASRURERNQKTKKIiIiIWoFJFBGRnVq9ejUUCgUOHjwodyhEdolJFBG1SENfzAsWLIBCoZAeLi4u6N+/P+bMmQO9Xl/vuaaHRqNBnz59MGPGDOTn59c7V25uLp599lmEhIRArVbDz88PCQkJ2LNnT5u81pbS6/V4++23MWjQILi5ucHZ2RkDBgzArFmzkJeXJ3d4FrFmzRosXbpU7jCIbIKj3AEQkf1Yvnw53NzccP36dfz444947733kJqaij179kChUEjt3nnnHYSGhqKiogK7d+/G8uXLsWXLFhw/fhwuLi4AgD179uDBBx8EADz99NPo378/tFotVq9ejfvuuw+ffPIJXnjhBVleZ0POnTuH2NhY5Obm4k9/+hOmTp0KlUqFo0eP4osvvsC3336LU6dOyR3mHVuzZg2OHz+Ol156Se5QiGTHJIqILOaRRx6Br68vAODZZ5/Fww8/jPXr12Pfvn2IjIyU2j3wwAMYOnQogNoEycfHBx999BG+++47jB8/HteuXcMjjzwCZ2dn7NmzBz179pSem5iYiLi4OLz00ksIDw/HyJEj2/ZFNqCmpgZ//OMfkZ+fj507dyIqKsps/3vvvYcPPvhApuiIyFo4nEdEVhMdHQ0AOH/+fIva/e1vf4NWq8XixYvNEigAcHZ2xldffQWFQoF33nnntjGUlZXhlVdeQXBwMNRqNfr27YsPP/wQQgizdgqFAjNmzMCGDRswYMAAqNVq3HXXXdi6dettz/Hf//4XR44cwVtvvVUvgQIAd3d3vPfee2bbkpOTER4eDmdnZ/j6+uKJJ57A5cuXzdo89dRTcHNzQ25uLn7729/Czc0NXbp0wbJlywAAx44dQ3R0NFxdXdG9e3esWbOmwfjKy8sxbdo0+Pj4wN3dHRMnTsS1a9fM2nz33XeIj49HUFAQ1Go1evbsiXfffRcGg0FqM2bMGGzevBkXLlyQhmRDQkJu+/4Q2SteiSIiqzl79iwAwMfHp0Xtvv/+e2g0Gjz66KMNtg8NDUVUVBRSU1Nx48YNODs7N9hOCIGHHnoIO3bswJQpUzB48GD873//w2uvvYbLly/j448/Nmu/e/durF+/Hs8//zw6deqETz/9FA8//DByc3ObfA0bN24EADz55JNNvk6T1atXY/LkyRg2bBiSkpKQn5+PTz75BHv27MHhw4fh6ekptTUYDHjggQcwatQoLFq0CN988w1mzJgBV1dXvPXWW5gwYQL++Mc/YsWKFZg4cSIiIyMRGhpqdr4ZM2bA09MTCxYsQHZ2NpYvX44LFy5g586d0jDr6tWr4ebmhsTERLi5uSE1NRXz5s2DXq/H4sWLAQBvvfUWdDodLl26JL13bm5uzXrNRHZJEBG1wKpVqwQAceDAAWnb/PnzBQCRnZ0tCgsLxfnz58Xf/vY3oVarhb+/vygrKzN77vbt20VhYaG4ePGiWLt2rfDx8RHOzs7i0qVLQgghPD09xaBBg5qM48UXXxQAxNGjRxtts2HDBgFA/OUvfzHb/sgjjwiFQiHOnDkjbQMgVCqV2bYjR44IAOKzzz5rMpYhQ4YIDw+PJtuYVFVVCT8/PzFgwABx48YNafumTZsEADFv3jxp26RJkwQA8f7770vbrl27JpydnYVCoRBr166VtmdlZQkAYv78+dI20/sdHh4uqqqqpO2LFi0SAMR3330nbSsvL68X67Rp04SLi4uoqKiQtsXHx4vu3bs367US2TsO5xGRxfTt2xedO3dGaGgopk2bhl69emHz5s3SZHGT2NhYdO7cGcHBwXj88cfh5uaGb7/9Fl26dAEAlJaWolOnTk2ey7S/7t1/v7ZlyxY4ODjgxRdfNNv+yiuvQAiBH374oV5cdYcP7777bri7u+PcuXNNxqLX628br8nBgwdRUFCA559/HhqNRtoeHx+PsLAwbN68ud5znn76aenvnp6e6Nu3L1xdXc2u1PXt2xeenp4Nxjp16lQ4OTlJPz/33HNwdHTEli1bpG11r+aVlpaiqKgI9913H8rLy5GVldWs10bU0XA4j4gs5r///S/c3d3h5OSErl271pvPZLJs2TL06dMHjo6O8Pf3R9++faFU3urTderUCaWlpU2ey7S/qeTlwoULCAoKqtemX79+0v66unXrVu8YXl5e9eYP/VpzEq26MQG1Sc+vhYWFYffu3WbbNBoNOnfubLbNw8MDXbt2Nbvj0bS9oVh79+5t9rObmxsCAwORk5MjbTtx4gTmzJmD1NTUeompTqe7/Qsj6oCYRBGRxYwaNUq6O68pw4cPl+7Oa0i/fv1w+PBhVFZWQq1WN9jm6NGjcHJyqpcg3AkHB4cGt4tfTUL/tbCwMBw+fBgXL15EcHCwxeJpKqbWxtqQkpISjB49Gu7u7njnnXfQs2dPaDQaHDp0CLNmzYLRaGzxMYk6Ag7nEZHN+e1vf4uKigokJyc3uD8nJwc///wzoqOjG51UDgDdu3dHXl5evatapuGp7t27WyTe3/3udwCAr7/++rZtTefMzs6uty87O9tiMdV1+vRps5+vX7+OK1euSHfW7dy5E1evXsXq1asxc+ZM/Pa3v0VsbCy8vLzqHevXV7+IOjImUURkc6ZNmwY/Pz+89tpr9YbJKioqMHnyZAghMG/evCaP8+CDD8JgMODzzz832/7xxx9DoVDggQcesEi8jzzyCAYOHIj33nsPaWlp9faXlpbirbfeAgAMHToUfn5+WLFiBSorK6U2P/zwA06ePIn4+HiLxFTXypUrUV1dLf28fPly1NTUSK/fdFWr7lWsqqoq/PWvf613LFdXVw7vEd3E4Twisjk+Pj74z3/+g/j4eNxzzz31KpafOXMGn3zyyW0Lbf7ud7/D2LFj8dZbbyEnJweDBg3Cjz/+iO+++w4vvfRSo3O2WsrJyQnr169HbGwsRo0ahUcffRT33nsvnJyccOLECaxZswZeXl5477334OTkhA8++ACTJ0/G6NGjMX78eKnEQUhICF5++WWLxFRXVVUVYmJi8OijjyI7Oxt//etfERUVhYceeggAMHLkSHh5eWHSpEl48cUXoVAo8P/+3/9rcGgwPDwc69atQ2JiIoYNGwY3NzfpShxRR8Mkiohs0n333YejR4/i/fffR3JyMq5cuQIPDw+MHDkSX375ZYNFLX9NqVRi48aNmDdvHtatW4dVq1YhJCQEixcvxiuvvGLReHv16oXMzEx8/PHH+Pbbb7FhwwYYjUb06tULTz/9tNkdgk899RRcXFywcOFCzJo1C66urvjDH/6ADz74wKxGlKV8/vnn+OabbzBv3jxUV1dj/Pjx+PTTT6WhOR8fH2zatAmvvPIK5syZAy8vLzzxxBOIiYlBXFyc2bGef/55ZGZmYtWqVfj444/RvXt3JlHUYSlEa2YhEhEREXVwnBNFRERE1ApMooiIiIhagUkUERERUSswiSIiIiJqBSZRRERERK3AJIqIiIioFVgnyoqMRiPy8vLQqVMnLpVARETUTgghUFpaiqCgILPF0X+NSZQV5eXlWXwxUiIiImobFy9eRNeuXRvdzyTKijp16gSg9kNwd3eXORoiIiJqDr1ej+DgYOl7vDFMoqzINITn7u7OJIqIiKidud1UHE4sJyIiImoFJlFERERErcAkioiIiKgVmEQRERERtQKTKCIiIqJWYBJFRERE1ApMooiIiIhagUkUERERUSswiSIiIiJqBSZRRERERK1gE0nUsmXLEBISAo1Gg4iICOzfv7/J9snJyQgLC4NGo8HAgQOxZcsWaV91dTVmzZqFgQMHwtXVFUFBQZg4cSLy8vLMjlFcXIwJEybA3d0dnp6emDJlCq5fv27W5ujRo7jvvvug0WgQHByMRYsWWe5FExERUbsmexK1bt06JCYmYv78+Th06BAGDRqEuLg4FBQUNNh+7969GD9+PKZMmYLDhw8jISEBCQkJOH78OACgvLwchw4dwty5c3Ho0CGsX78e2dnZeOihh8yOM2HCBJw4cQLbtm3Dpk2bsGvXLkydOlXar9frMW7cOHTv3h0ZGRlYvHgxFixYgJUrV1rvzSAiIqL2Q8hs+PDhYvr06dLPBoNBBAUFiaSkpAbbP/rooyI+Pt5sW0REhJg2bVqj59i/f78AIC5cuCCEEOKXX34RAMSBAwekNj/88INQKBTi8uXLQggh/vrXvwovLy9RWVkptZk1a5bo27dvs1+bTqcTAIROp2v2c4iIiOj2Dl0oFmWV1VY5dnO/v2W9ElVVVYWMjAzExsZK25RKJWJjY5GWltbgc9LS0szaA0BcXFyj7QFAp9NBoVDA09NTOoanpyeGDh0qtYmNjYVSqUR6errUZtSoUVCpVGbnyc7OxrVr1xo8T2VlJfR6vdmDiIiILKui2oAJ/0jH4He24cLVMtnikDWJKioqgsFggL+/v9l2f39/aLXaBp+j1Wpb1L6iogKzZs3C+PHj4e7uLh3Dz8/PrJ2joyO8vb2l4zR2HtO+hiQlJcHDw0N6BAcHN9iOiIiIWm/fuasorzLAy8UJ3bxdZItD9jlR1lRdXY1HH30UQggsX77c6uebPXs2dDqd9Lh48aLVz0lERNTRpGbVzpuODvOHQqGQLQ5H2c4MwNfXFw4ODsjPzzfbnp+fj4CAgAafExAQ0Kz2pgTqwoULSE1Nla5CmY7x64nrNTU1KC4ulo7T2HlM+xqiVquhVqsbe7lERER0h4QQSDlZ+x0e28/vNq2tS9YrUSqVCuHh4UhJSZG2GY1GpKSkIDIyssHnREZGmrUHgG3btpm1NyVQp0+fxvbt2+Hj41PvGCUlJcjIyJC2paamwmg0IiIiQmqza9cuVFdXm52nb9++8PLyav2LJiIiolbLzi/F5ZIbUDsqMbKnr6yxyD6cl5iYiL///e/46quvcPLkSTz33HMoKyvD5MmTAQATJ07E7NmzpfYzZ87E1q1bsWTJEmRlZWHBggU4ePAgZsyYAaA2gXrkkUdw8OBBfPPNNzAYDNBqtdBqtaiqqgIA9OvXD/fffz+eeeYZ7N+/H3v27MGMGTPw+OOPIygoCADw5z//GSqVClOmTMGJEyewbt06fPLJJ0hMTGzjd4iIiIhMTFeh7u3lC2eVg7zBWOXewBb67LPPRLdu3YRKpRLDhw8X+/btk/aNHj1aTJo0yaz9v//9b9GnTx+hUqnEXXfdJTZv3iztO3/+vADQ4GPHjh1Su6tXr4rx48cLNzc34e7uLiZPnixKS0vNznPkyBERFRUl1Gq16NKli1i4cGGLXhdLHBAREVnWH5btFt1nbRJf78ux2jma+/2tEEII+VI4+6bX6+Hh4QGdTmc2J4uIiIha7ur1Sgx9bzuEANJmRyPQw9kq52nu97fsw3lEREREzbEzuxBCAP0D3a2WQLUEkygiIiJqF1Kyau+Sl/uuPBMmUURERGTzqmqM2HWqCAAQ3c//Nq3bBpMoIiIisnkHcopxvbIGvm5q3N3FQ+5wADCJIiIionZg+8naobzosM5QKuWrUl4XkygiIiKyaaJOlfLoMNsYygOYRBEREZGNO1tYhtzicqgclLivt7xVyutiEkVEREQ2LeXmUN6Inj5wVcu67K8ZJlFERERk01KyaofyYsJso7SBCZMoIiIislkl5VXIuHANABDNJIqIiIioeX46VQiDUaCvfycEe7vIHY4ZJlFERERks6S78mykSnldTKKIiIjIJtUYjNiZbZvzoQAmUURERGSjDl64Bn1FDbxcnDCkm5fc4dTDJIqIiIhsUurNu/LG9vWDg41UKa+LSRQRERHZJFN9KFucDwUwiSIiIiIblFNUhrOFZXBUKjCqT2e5w2kQkygiIiKyOaYCm8NDveGucZI5moYxiSIiIiKbk5p1cyjPBu/KM2ESRURERDZFX1GN9HPFAIDYfv4yR9M4JlFERERkU34+VYQao0CPzq4I8XWVO5xGMYkiIiIim5JycyjPlq9CAUyiiIiIyIYYjAI7swsB2PZ8KIBJFBEREdmQzIvXUFxWBXeNI8K7216V8rqYRBEREZHNMC04PKavH5wcbDtNse3oiIiIqEMxJVExNlqlvC4mUURERGQTLhaXIzu/FA5KBUbbaJXyuphEERERkU3YkV17FSq8uxc8XVQyR3N7TKKIiIjIJmw3DeXZ+F15JkyiiIiISHZllTXYd/YqgPYxHwpgEkVEREQ24OfTRagyGNHdxwU9O7vJHU6zyJ5ELVu2DCEhIdBoNIiIiMD+/fubbJ+cnIywsDBoNBoMHDgQW7ZsMdu/fv16jBs3Dj4+PlAoFMjMzDTbn5OTA4VC0eAjOTlZatfQ/rVr11rsdRMREdEtdRccVigUMkfTPLImUevWrUNiYiLmz5+PQ4cOYdCgQYiLi0NBQUGD7ffu3Yvx48djypQpOHz4MBISEpCQkIDjx49LbcrKyhAVFYUPPvigwWMEBwfjypUrZo+3334bbm5ueOCBB8zarlq1yqxdQkKCxV47ERER1TIaBVKzaquUx4TZ9lIvdSmEEEKuk0dERGDYsGH4/PPPAQBGoxHBwcF44YUX8MYbb9Rr/9hjj6GsrAybNm2Sto0YMQKDBw/GihUrzNrm5OQgNDQUhw8fxuDBg5uMY8iQIbjnnnvwxRdfSNsUCgW+/fbbO0qc9Ho9PDw8oNPp4O7u3urjEBER2bPMiyVIWLYHbmpHHJr7G6gc5R0oa+73t2xRVlVVISMjA7GxsbeCUSoRGxuLtLS0Bp+TlpZm1h4A4uLiGm3fHBkZGcjMzMSUKVPq7Zs+fTp8fX0xfPhwfPnll7hdvllZWQm9Xm/2ICIioqalnqwdyhvVx1f2BKolHOU6cVFREQwGA/z9zS/b+fv7Iysrq8HnaLXaBttrtdpWx/HFF1+gX79+GDlypNn2d955B9HR0XBxccGPP/6I559/HtevX8eLL77Y6LGSkpLw9ttvtzoWIiKijiglq3YaT3Q7GsoDZEyibMGNGzewZs0azJ07t96+utuGDBmCsrIyLF68uMkkavbs2UhMTJR+1uv1CA4OtmzQREREduSK7gZO5OmhUABj+9p+lfK6ZLtm5uvrCwcHB+Tn55ttz8/PR0BAQIPPCQgIaFH72/nPf/6D8vJyTJw48bZtIyIicOnSJVRWVjbaRq1Ww93d3exBREREjUu9eRVqSLAnfNzUMkfTMrIlUSqVCuHh4UhJSZG2GY1GpKSkIDIyssHnREZGmrUHgG3btjXa/na++OILPPTQQ+jc+faZb2ZmJry8vKBWt68PmIiIyJalSgsOt6+hPEDm4bzExERMmjQJQ4cOxfDhw7F06VKUlZVh8uTJAICJEyeiS5cuSEpKAgDMnDkTo0ePxpIlSxAfH4+1a9fi4MGDWLlypXTM4uJi5ObmIi8vDwCQnZ0NoPYqVt0rVmfOnMGuXbvq1ZkCgO+//x75+fkYMWIENBoNtm3bhvfffx+vvvqq1d4LIiKijuZGlQG7zxQBaD9VyuuSNYl67LHHUFhYiHnz5kGr1WLw4MHYunWrNHk8NzcXSuWti2UjR47EmjVrMGfOHLz55pvo3bs3NmzYgAEDBkhtNm7cKCVhAPD4448DAObPn48FCxZI27/88kt07doV48aNqxeXk5MTli1bhpdffhlCCPTq1QsfffQRnnnmGUu/BURERB3W3rNFqKwxoounM/r6d5I7nBaTtU6UvWOdKCIiosa9+e0xrEnPxZMjuuPdhAG3f0Ibsfk6UURERNRxCSHqzIdqf0N5AJMoIiIiksGJPD20+go4OzlgRA8fucNpFSZRRERE1OZMpQ2ievtC4+QgczStwySKiIiI2lzKzaVeYtvpUB7AJIqIiIjaWEFpBY5c0gEAxvZlEkVERETULDuzCgEAd3f1gJ+7RuZoWo9JFBEREbWp7TeH8mLa2YLDv8YkioiIiNpMRXX7rlJeF5MoIiIiajPp54tRXmWAv7sadwW170LUTKKIiIiozZjuyosO84dCoZA5mjvDJIqIiIjahBACKaYq5WHteygPYBJFREREbeRU/nVcLrkBtaMS9/bylTucO8YkioiIiNqE6a68e3v5wlnVPquU18UkioiIiNqEaamXaDsYygOYRBEREVEbKC6rwqHcawDaf2kDEyZRREREZHU7sgogBNA/0B2BHs5yh2MRTKKIiIjI6kxDefZyFQpgEkVERERWVlVjxK5TtevlxfRr30u91MUkioiIiKzqQE4xSitr4Oumwt1dPOQOx2KYRBEREZFVmQpsju3rB6WyfVcpr4tJFBEREVmNEAIpWbX1oexpKA9gEkVERERWdLawDBeulkPloERU7/ZfpbwuJlFERERkNak3r0JF9PCGm9pR5mgsi0kUERERWc32m/OhYu1sKA9gEkVERERWUlJehYwLtVXK7WWpl7qYRBEREZFV/HSqEAajQB9/NwR7u8gdjsUxiSIiIiKrMJU2sLe78kyYRBEREZHF1RiM2Jl9M4myw6E8gEkUERERWUHGhWvQV9TAy8UJQ7p5yR2OVTCJIiIiIotLybpVpdzBjqqU18UkioiIiCwu5WRtfajofvY5lAfYQBK1bNkyhISEQKPRICIiAvv372+yfXJyMsLCwqDRaDBw4EBs2bLFbP/69esxbtw4+Pj4QKFQIDMzs94xxowZA4VCYfZ49tlnzdrk5uYiPj4eLi4u8PPzw2uvvYaampo7fr1ERET2LqeoDGcLy+CoVGBUn85yh2M1siZR69atQ2JiIubPn49Dhw5h0KBBiIuLQ0FBQYPt9+7di/Hjx2PKlCk4fPgwEhISkJCQgOPHj0ttysrKEBUVhQ8++KDJcz/zzDO4cuWK9Fi0aJG0z2AwID4+HlVVVdi7dy+++uorrF69GvPmzbPMCyciIrJjpqG84aHecNc4yRyN9SiEEEKuk0dERGDYsGH4/PPPAQBGoxHBwcF44YUX8MYbb9Rr/9hjj6GsrAybNm2Sto0YMQKDBw/GihUrzNrm5OQgNDQUhw8fxuDBg832jRkzBoMHD8bSpUsbjOuHH37Ab3/7W+Tl5cHfv/a2zBUrVmDWrFkoLCyESqVq1uvT6/Xw8PCATqeDu7t7s55DRETU3k34xz7sOXMVc+L74en7esgdTos19/tbtitRVVVVyMjIQGxs7K1glErExsYiLS2tweekpaWZtQeAuLi4Rts35ZtvvoGvry8GDBiA2bNno7y83Ow8AwcOlBIo03n0ej1OnDjR6DErKyuh1+vNHkRERB1JaUU10s8VA7Df+lAmsq0EWFRUBIPBYJaoAIC/vz+ysrIafI5Wq22wvVarbdG5//znP6N79+4ICgrC0aNHMWvWLGRnZ2P9+vVNnse0rzFJSUl4++23WxQLERGRPdl1qgg1RoEenV0R6usqdzhWZV/LKTfT1KlTpb8PHDgQgYGBiImJwdmzZ9GzZ89WH3f27NlITEyUftbr9QgODr6jWImIiNqTlKzau/LstcBmXbIN5/n6+sLBwQH5+flm2/Pz8xEQENDgcwICAlrUvrkiIiIAAGfOnGnyPKZ9jVGr1XB3dzd7EBERdRQGo8DO7EIA9j+UB8iYRKlUKoSHhyMlJUXaZjQakZKSgsjIyAafExkZadYeALZt29Zo++YylUEIDAyUznPs2DGzuwS3bdsGd3d39O/f/47ORUREZK8yL15DcVkV3DWOCO9un1XK65J1OC8xMRGTJk3C0KFDMXz4cCxduhRlZWWYPHkyAGDixIno0qULkpKSAAAzZ87E6NGjsWTJEsTHx2Pt2rU4ePAgVq5cKR2zuLgYubm5yMvLAwBkZ2cDqL2CFBAQgLNnz2LNmjV48MEH4ePjg6NHj+Lll1/GqFGjcPfddwMAxo0bh/79++PJJ5/EokWLoNVqMWfOHEyfPh1qtbot3yIiIqJ2w7Tg8Oi+fnBykL0UpfUJmX322WeiW7duQqVSieHDh4t9+/ZJ+0aPHi0mTZpk1v7f//636NOnj1CpVOKuu+4SmzdvNtu/atUqAaDeY/78+UIIIXJzc8WoUaOEt7e3UKvVolevXuK1114TOp3O7Dg5OTnigQceEM7OzsLX11e88sororq6ukWvTafTCQD1jk1ERGSP4j7+SXSftUlsOHxJ7lDuSHO/v2WtE2XvWCeKiIg6ikvXyhH1wQ4oFcChub+Bp0vzairaIpuvE0VERET2I/VmlfKh3b3bdQLVEkyiiIiI6I6Z5kPF2PGCw7/GJIqIiIjuSFllDdLOXgXAJIqIiIio2XafKUKVwYhu3i7o2dlN7nDaDJMoIiIiuiOpdYbyFAqFzNG0HSZRRERE1GpGo0DKzUnlMWH2X6W8LiZRRERE1GrHLutQdL0SbmpHDA/1ljucNsUkioiIiFrNdBVqVB9fqBw7VlrRsV4tERERWVTKyXwAQHQHG8oDmEQRERFRK2l1FTiRp4dCAYzp21nucNockygiIiJqFVOV8iHBnvB1U8scTdtjEkVEREStYhrKi+nX8YbyACZRRERE1Ao3qgzYfaYIABAd1nGqlNfFJIqIiIhabO/ZIlTWGNHF0xlhAZ3kDkcWTKKIiIioxUylDaLDOlaV8rqYRBEREVGLCCGkpV6iO9CCw7/GJIqIiIha5ESeHlp9BZydHBDZw0fucGTDJIqIiIhaxFTaIKq3LzRODjJHIx8mUURERNQitxYc7rhDeQCTKCIiImqBgtIKHLlYAqDjljYwYRJFREREzbYzqxAAcHdXD/i5a2SORl5MooiIiKjZUrJMCw537KtQAJMoIiIiaqaKagN+Pl1bpTy2gy71UheTKCIiImqW9PPFKK8ywN9djbuC3OUOR3ZMooiIiKhZUk+ahvL8O2yV8rqYRBEREdFtCSGw/SRLG9TFJIqIiIhu61T+dVwuuQG1oxL39vKVOxybwCSKiIiIbst0V969vXzhrOq4VcrrYhJFREREt5ViWnCYQ3kSJlFERETUpOKyKhzKvQaASVRdTKKIiIioSTuzCyAE0D/QHUGeznKHYzNkT6KWLVuGkJAQaDQaREREYP/+/U22T05ORlhYGDQaDQYOHIgtW7aY7V+/fj3GjRsHHx8fKBQKZGZmmu0vLi7GCy+8gL59+8LZ2RndunXDiy++CJ1OZ9ZOoVDUe6xdu9Yir5mIiKg9MQ3lxfTjVai6ZE2i1q1bh8TERMyfPx+HDh3CoEGDEBcXh4KCggbb7927F+PHj8eUKVNw+PBhJCQkICEhAcePH5falJWVISoqCh988EGDx8jLy0NeXh4+/PBDHD9+HKtXr8bWrVsxZcqUem1XrVqFK1euSI+EhASLvG4iIqL2oqrGiF2natfL41CeOYUQQsh18oiICAwbNgyff/45AMBoNCI4OBgvvPAC3njjjXrtH3vsMZSVlWHTpk3SthEjRmDw4MFYsWKFWducnByEhobi8OHDGDx4cJNxJCcn44knnkBZWRkcHR0B1F6J+vbbb+8ocdLr9fDw8IBOp4O7Oyu7EhFR+7P3TBH+/I90+LqpsP/NWCiV9l9ks7nf37JdiaqqqkJGRgZiY2NvBaNUIjY2FmlpaQ0+Jy0tzaw9AMTFxTXavrlMb5IpgTKZPn06fH19MXz4cHz55Ze4Xb5ZWVkJvV5v9iAiImrPTAU2x/b16xAJVEs43r6JdRQVFcFgMMDf33wBQ39/f2RlZTX4HK1W22B7rVZ7R3G8++67mDp1qtn2d955B9HR0XBxccGPP/6I559/HtevX8eLL77Y6LGSkpLw9ttvtzoWIiIiWyKEkOpDcT5UfbIlUbZAr9cjPj4e/fv3x4IFC8z2zZ07V/r7kCFDUFZWhsWLFzeZRM2ePRuJiYlmxw8ODrZ43ERERG3hXFEZLlwth8pBiajeneUOx+bINpzn6+sLBwcH5Ofnm23Pz89HQEBAg88JCAhoUfumlJaW4v7770enTp3w7bffwsnJqcn2ERERuHTpEiorKxtto1ar4e7ubvYgIiJqr1JuLjgc0cMbbuoOfd2lQbIlUSqVCuHh4UhJSZG2GY1GpKSkIDIyssHnREZGmrUHgG3btjXavjF6vR7jxo2DSqXCxo0bodFobvuczMxMeHl5Qa1Wt+hcRERE7VUKFxxukqxpZWJiIiZNmoShQ4di+PDhWLp0KcrKyjB58mQAwMSJE9GlSxckJSUBAGbOnInRo0djyZIliI+Px9q1a3Hw4EGsXLlSOmZxcTFyc3ORl5cHAMjOzgZQexUrICBASqDKy8vx9ddfm00A79y5MxwcHPD9998jPz8fI0aMgEajwbZt2/D+++/j1Vdfbcu3h4iISDa68mocvFBbpTymn/9tWndMsiZRjz32GAoLCzFv3jxotVoMHjwYW7dulSaP5+bmQqm8dbFs5MiRWLNmDebMmYM333wTvXv3xoYNGzBgwACpzcaNG6UkDAAef/xxAMD8+fOxYMECHDp0COnp6QCAXr16mcVz/vx5hISEwMnJCcuWLcPLL78MIQR69eqFjz76CM8884zV3gsiIiJbsvNUAQxGgT7+bgj2dpE7HJska50oe8c6UURE1F7NXHsY32Xm4dnRPfHGA2Fyh9OmbL5OFBEREdmmGoMRO7Nrq5THsrRBo5hEERERkZmMC9egu1ENTxcnDOnmJXc4NotJFBEREZlJzbpVpdyBVcobxSSKiIiIzKTcTKJYpbxpTKKIiIhIcuFqGc4UXIejUoH7WKW8SUyiiIiISGIqsDksxBsezk2v5tHRMYkiIiIiCRccbr5WJ1FVVVXIzs5GTU2NJeMhIiIimZRWVCP9XDEAVilvjhYnUeXl5ZgyZQpcXFxw1113ITc3FwDwwgsvYOHChRYPkIiIiNrGz6eLUGMU6OHrilBfV7nDsXktTqJmz56NI0eOYOfOnWYL98bGxmLdunUWDY6IiIjazvaTHMpriRavnbdhwwasW7cOI0aMgEJxq3bEXXfdhbNnz1o0OCIiImobBqOQqpRHh3EorzlafCWqsLAQfn71M9SysjKzpIqIiIjaj8yLJSguq4K7xhFDQ1ilvDlanEQNHToUmzdvln42JU7/+Mc/EBkZabnIiIiIqM2k3BzKG93XD04OvHm/OVo8nPf+++/jgQcewC+//IKamhp88skn+OWXX7B371789NNP1oiRiIiIrMy01EtMGOdDNVeLU82oqChkZmaipqYGAwcOxI8//gg/Pz+kpaUhPDzcGjESERGRFV26Vo4sbSmUCmBMX1Ypb64WX4kCgJ49e+Lvf/+7pWMhIiIiGZiuQg3t7g1PF5XM0bQfLb4S5eDggIKCgnrbr169CgcHB4sERURERG3HtNRLNEsbtEiLkyghRIPbKysroVIxeyUiImpPyiprkHb2KgAglklUizR7OO/TTz8FUHs33j/+8Q+4ublJ+wwGA3bt2oWwsDDLR0hERERWs/tMEaoMRnTzdkHPzm63fwJJmp1EffzxxwBqr0StWLHCbOhOpVIhJCQEK1assHyEREREZDWppqG8MD/We2yhZidR58+fBwCMHTsW69evh5cXC3ERERG1Z0ajQGp2bRIVywWHW6zFd+ft2LHDGnEQERFRGzt2WYfC0kq4qhwwPNRb7nDanVaVOLh06RI2btyI3NxcVFVVme376KOPLBIYERERWVfKzdIGo/p0hsqRVcpbqsVJVEpKCh566CH06NEDWVlZGDBgAHJyciCEwD333GONGImIiMgKUrNql3qJ4VBeq7Q47Zw9ezZeffVVHDt2DBqNBv/9739x8eJFjB49Gn/605+sESMRERFZmFZXgeOX9VCwSnmrtTiJOnnyJCZOnAgAcHR0xI0bN+Dm5oZ33nkHH3zwgcUDJCIiIsszVSkfHOwJXze1zNG0Ty1OolxdXaV5UIGBgTh79qy0r6ioyHKRERERkdWYhvJ4V17rtXhO1IgRI7B7927069cPDz74IF555RUcO3YM69evx4gRI6wRIxEREVlQRbUBu8/UXviIDmOV8tZqcRL10Ucf4fr16wCAt99+G9evX8e6devQu3dv3plHRETUDuw9W4SKaiOCPDQIC+gkdzjtVouSKIPBgEuXLuHuu+8GUDu0xyrlRERE7YtpweGYfv6sUn4HWjQnysHBAePGjcO1a9esFQ8RERFZkRBCmlQezQWH70iLJ5YPGDAA586ds1gAy5YtQ0hICDQaDSIiIrB///4m2ycnJyMsLAwajQYDBw7Eli1bzPavX78e48aNg4+PDxQKBTIzM+sdo6KiAtOnT4ePjw/c3Nzw8MMPIz8/36xNbm4u4uPj4eLiAj8/P7z22muoqam549dLREQkp1+u6HFFVwFnJwdE9vCRO5x2rcVJ1F/+8he8+uqr2LRpE65cuQK9Xm/2aIl169YhMTER8+fPx6FDhzBo0CDExcWhoKCgwfZ79+7F+PHjMWXKFBw+fBgJCQlISEjA8ePHpTZlZWWIiopqstzCyy+/jO+//x7Jycn46aefkJeXhz/+8Y/SfoPBgPj4eFRVVWHv3r346quvsHr1asybN69Fr4+IiMjWmBYcjurtC42Tg8zRtHOihRQKhfRQKpXSw/RzSwwfPlxMnz5d+tlgMIigoCCRlJTUYPtHH31UxMfHm22LiIgQ06ZNq9f2/PnzAoA4fPiw2faSkhLh5OQkkpOTpW0nT54UAERaWpoQQogtW7YIpVIptFqt1Gb58uXC3d1dVFZWNvv16XQ6AUDodLpmP4eI7lxFdY3cIRDZrIc+3y26z9ok/pV+Qe5QbFZzv79lW4C4qqoKGRkZmD17trRNqVQiNjYWaWlpDT4nLS0NiYmJZtvi4uKwYcOGZp83IyMD1dXViI2NlbaFhYWhW7duSEtLw4gRI5CWloaBAwfC3/9W7Yy4uDg899xzOHHiBIYMGdLgsSsrK1FZWSn93NIrc0R055IPXsRr/zmKZX++B/F3B8odDpFNKSytxJGLJQBY2sASWpxEjR492iInLioqgsFgMEtUAMDf3x9ZWVkNPker1TbYXqvVNvu8Wq0WKpUKnp6ejR6nsfOY9jUmKSkJb7/9drNjISLL+yY9FwCwZv8FJlFEv7Iju3Yob2AXD/i5a2SOpv3jks0WNHv2bOh0Oulx8eJFuUMi6lAKSytx5FIJACD9XDH0FdXyBkRkY1JOmhYc5lUoS5AtifL19YWDg0O9u+Ly8/MREBDQ4HMCAgJa1L6xY1RVVaGkpKTR4zR2HtO+xqjVari7u5s9iKjt7MgugBC1f68xCvx8iktREZlU1hjw8+na/xMxYVzqxRJkS6JUKhXCw8ORkpIibTMajUhJSUFkZGSDz4mMjDRrDwDbtm1rtH1DwsPD4eTkZHac7Oxs5ObmSseJjIzEsWPHzO4S3LZtG9zd3dG/f/9mn4uI2papl+2mdjT7mYiAfeeKUV5lgL+7GgO6sJNvCS2eE2VJiYmJmDRpEoYOHYrhw4dj6dKlKCsrw+TJkwEAEydORJcuXZCUlAQAmDlzJkaPHo0lS5YgPj4ea9euxcGDB7Fy5UrpmMXFxcjNzUVeXh6A2gQJqL2CFBAQAA8PD0yZMgWJiYnw9vaGu7s7XnjhBURGRkpr/40bNw79+/fHk08+iUWLFkGr1WLOnDmYPn061GqudE1ki+r2sl+K7Y2/bD6JHdkFMBgFHJSsyEyUerNTER3mxyrlFiLrnKjHHnsMH374IebNm4fBgwcjMzMTW7dulSZx5+bm4sqVK1L7kSNHYs2aNVi5ciUGDRqE//znP9iwYQMGDBggtdm4cSOGDBmC+Ph4AMDjjz+OIUOGmC1P8/HHH+O3v/0tHn74YYwaNQoBAQFYv369tN/BwQGbNm2Cg4MDIiMj8cQTT2DixIl45513rP2WEFErpdfpZU+MDIG7xhHXyquReZErLBAJIZBys0o5h/IsRyGEaQZB8+Tn5+PVV19FSkoKCgoK8OunGwwGiwbYnun1enh4eECn03F+FJGVzf/uOL5Ku4Dxw4OR9Me78cK/DuP7I3l4bkxPzLo/TO7wiGSVrS1F3NJdUDsqkTlvHJxVLLLZlOZ+f7d4OO+pp55Cbm4u5s6di8DAQF4SJCLZNdTLju3nh++P5CH1ZAGTKOrwUrJqh/JG9vRhAmVBLU6idu/ejZ9//hmDBw+2QjhERC13uuA6Ll27AbWjEvf28gUAjO7TGUoFkJ1fiovF5Qj2dpE5SiL5mJZ6ienHoTxLavGcqODg4HpDeEREctp+sn4v29NFhaHdvQFAWrGeqCMqLqvCodzauYGsUm5ZLU6ili5dijfeeAM5OTlWCIeIqOUa62WbCgqmMImiDmxndgGMAugX6I4gT2e5w7ErLR7Oe+yxx1BeXo6ePXvCxcUFTk5OZvuLi4stFhwR0e001cuO6eeHpB+ysO/sVZRV1sBVLWtVFyJZmDoRsaxSbnEt/o2ydOlSK4RBRNQ6TfWye3Z2QzdvF+QWl+Pn00W4f0DzVzcgsgdVNUbsyi4EwKE8a2hxEjVp0iRrxEFE1CpN9bIVCgVi+vlh1Z4cpGblM4miDudgTjFKK2vg66bCoK6ecodjd5qVROn1eqlOgl6vb7It6yERUVupNty+lx0T5n8ziSqE0SigZPVy6kBMnYyxff34b98KmpVEeXl54cqVK/Dz84Onp2eDtaGEEFAoFCy2SURt5sD52/eyh4d6w03tiKLrlTh6WYfBwQ23I7I3Qghp/cgYzoeyimYlUampqfD29pb+zgKbRGQLmtPLVjkqMaqPL7Yc0yL1ZD6TKOowzhWVIedqOVQOSkT17ix3OHapWUnU6NGjcf78eYSGhmLMmDFWDomIqHlM9Z9u18uODvPHlmNapGQVIHFc37YIjUh2ptIfET1qr8aS5TW7TlTPnj0RGhqK//u//8PXX3+NS5cuWTMuIqImnS28jvNFZc3qZY/p2xkKBXAiT48ruhttFCGRvExFaGN4V57VNDuJSk1NxaRJk3Du3Dk888wz6N69O3r37o1p06Zh7dq1yM/Pt2acRERmWtLL9nVTY8jNYTxWL6eOQFdejYMXauuncakX62n29b0xY8ZIQ3kVFRXYu3cvdu7ciZ07d+Krr75CdXU1wsLCcOLECWvFSkQkMS2o2txedkw/fxzKLUHqyQJMiOhuzdCIZPfT6UIYjAJ9/N24bqQVtXjZFwDQaDSIjo7GnDlz8Pbbb+PFF1+Em5sbsrKyLB0fEVE9uvJqHMhpWS/bVAJh95ki3KjiXcRk30x35UWH8SqUNbUoiaqqqsKuXbvw9ttvY+zYsfD09MSzzz6La9eu4fPPP8f58+etFScRkaQ1veywgE7o4umMyhoj9p4tsnKERPKpMRix82b9NJY2sK5mD+dFR0cjPT0doaGhGD16NKZNm4Y1a9YgMDDQmvEREdWT2opetkKhQHSYH/7fvgtIySrgPBGyW4dyS6C7UQ1PFyfc081L7nDsWrOvRP3888/w8fFBdHQ0YmJi8Jvf/IYJFBG1uRqDETta2cuOvtk+9WQBhBAWj43IFpiG8sb29YMDq5RbVbOTqJKSEqxcuRIuLi744IMPEBQUhIEDB2LGjBn4z3/+g8LCQmvGSUQE4M562ZE9fODs5ACtvgIn8ppewoqovTIVoeWCw9bX7CTK1dUV999/PxYuXIj09HQUFRVh0aJFcHFxwaJFi9C1a1cMGDDAmrESEUl35bWml61xckBUb18ALHVA9unC1TKcKbgOR6UCo/qwSrm1teruPKA2qfL29oa3tze8vLzg6OiIkydPWjI2IqJ6Uk7eWS/bVBLBNORBZE9M/z+GhXjDw9lJ5mjsX7MnlhuNRhw8eBA7d+7Ejh07sGfPHpSVlaFLly4YO3Ysli1bhrFjx1ozViLq4CzRyzYlX0cu6VBQWgG/ThpLhkgkq+YuhUSW0ewkytPTE2VlZQgICMDYsWPx8ccfY8yYMejZs6c14yMikpi+IO6kl+3nrsHdXT1w9JIOO7MK8eiwYEuGSCSb0opqpJ+/CoBVyttKs5OoxYsXY+zYsejTp4814yEiapRpqOJOe9kxYf44ekmH7SfzmUSR3fj5dBGqDQI9fF0R6usqdzgdQrPnRE2bNo0JFBHJxpK9bFMStvtMESqqWb2c7IOlOhnUfK2eWE5E1JYs2cu+K8gd/u5qlFcZsO/cVQtFSCQfg1FgR7bppgsO5bUVJlFE1C5YspddW7289ouGpQ7IHmReLEFxWRU6aRwxNIRVytsKkygisnkGo8BOC/eyb5U6YPVyav9Sb9ZPG9PXD04O/GpvK3ynicjmZV4swVUL97Lv7eULtaMSl0tuIDu/1CLHJJKLdKWWVcrbFJMoIrJ51uhlO6sccG+v2urlpi8govbo0rVyZGlLoVQAo1mlvE0xiSIim2etXrap8CbnRVF7tuPmv9+h3b3h5aqSOZqOxSaSqGXLliEkJAQajQYRERHYv39/k+2Tk5MRFhYGjUaDgQMHYsuWLWb7hRCYN28eAgMD4ezsjNjYWJw+fVrav3PnTigUigYfBw4cAADk5OQ0uH/fvn2WfwOIqFHW7GWbkqhDuddw9XqlRY9N1Fa2m5ZCYmmDNid7ErVu3TokJiZi/vz5OHToEAYNGoS4uDgUFDTcM9y7dy/Gjx+PKVOm4PDhw0hISEBCQgKOHz8utVm0aBE+/fRTrFixAunp6XB1dUVcXBwqKioAACNHjsSVK1fMHk8//TRCQ0MxdOhQs/Nt377drF14eLj13gwiqseavewgT2f0D3SHEMDO7EKLHpuoLZRV1iDt7M36aZwP1eZkT6I++ugjPPPMM5g8eTL69++PFStWwMXFBV9++WWD7T/55BPcf//9eO2119CvXz+8++67uOeee/D5558DqL0KtXTpUsyZMwe///3vcffdd+Of//wn8vLysGHDBgCASqVCQECA9PDx8cF3332HyZMnQ6EwXxXex8fHrK2TExd0JGpLKVnW7WWbSiZwSI/aoz1nilBlMKKbtwt6+bnJHU6HI2sSVVVVhYyMDMTGxkrblEolYmNjkZaW1uBz0tLSzNoDQFxcnNT+/Pnz0Gq1Zm08PDwQERHR6DE3btyIq1evYvLkyfX2PfTQQ/Dz80NUVBQ2btzY5OuprKyEXq83exBR65VX1WCvlXvZpiG9n04VoqrGaJVzEFmLab5gdJhfvYsAZH2yJlFFRUUwGAzw9zev++Lv7w+tVtvgc7RabZPtTX+25JhffPEF4uLi0LVrV2mbm5sblixZguTkZGzevBlRUVFISEhoMpFKSkqCh4eH9AgO5ppcRHdi9+kiVNVYt5c9qKsnfN1UuF5ZgwM5xVY5B5E1GI0Cqdlc6kVOsg/nye3SpUv43//+hylTppht9/X1RWJiIiIiIjBs2DAsXLgQTzzxBBYvXtzosWbPng2dTic9Ll68aO3wieyaaYjNmr1spVKBsX1vFd4kai+O5+lQWFoJV5UDIkJ95A6nQ5I1ifL19YWDgwPy8/PNtufn5yMgIKDB5wQEBDTZ3vRnc4+5atUq+Pj44KGHHrptvBEREThz5kyj+9VqNdzd3c0eRNQ6RqOQ5kNZu5dtOn5KVj6rl1O7Yborb1SfzlA5dvhrIrKQ9V1XqVQIDw9HSkqKtM1oNCIlJQWRkZENPicyMtKsPQBs27ZNah8aGoqAgACzNnq9Hunp6fWOKYTAqlWrMHHixGZNGM/MzERgYGCzXx8RtV5b9rKjeneGykGJC1fLcbawzKrnIrIUUxHaaN6VJxtHuQNITEzEpEmTMHToUAwfPhxLly5FWVmZNMl74sSJ6NKlC5KSkgAAM2fOxOjRo7FkyRLEx8dj7dq1OHjwIFauXAmgdmHRl156CX/5y1/Qu3dvhIaGYu7cuQgKCkJCQoLZuVNTU3H+/Hk8/fTT9eL66quvoFKpMGTIEADA+vXr8eWXX+If//iHFd8NIjJJacNetpvaERE9vPHz6SKkZuXzLieyeVpdBY5f1kOhAMYyiZKN7EnUY489hsLCQsybNw9arRaDBw/G1q1bpYnhubm5UCpv/QIdOXIk1qxZgzlz5uDNN99E7969sWHDBgwYMEBq8/rrr6OsrAxTp05FSUkJoqKisHXrVmg0GrNzf/HFFxg5ciTCwsIajO3dd9/FhQsX4OjoiLCwMKxbtw6PPPKIFd4FIvq1lDbuZceE+eHn00XYfrIAU0f1bJNzErWWab7g4GBP+LqpZY6m41IITgCwGr1eDw8PD+h0Os6PImoBra4CI5JSoFAAB96KbZMviYvF5bhv0Q44KBXImBMLTxcun0G26+mvDmD7yQK8Oq4PZkT3ljscu9Pc72/ORCMim7Mju+172cHeLujj7waDUeCnU6xeTrarotqA3WeKAAAx/fxv05qsiUkUEdmclJO1Q3ltvYxFdJj/zfOz1AHZrr1ni1BRbUSQhwZhAZ3kDqdDYxJFRDZFzl527M1SBzuzC1BjYPVysk0pdRYcZpVyeTGJIiKbknb2qmy97CHdvODp4gR9RQ0yLlxr03MTNYcQQppUzqE8+TGJIiKbsv3mUJ4cvWyHutXLuSAx2aBfruhxRVcBZycHRPZglXK5MYkiIpthC71sqXr5yfzbtCRqe6k3h/Lu7eULjZODzNEQkygishknr5TK3su+r3dnOCoVOFtYhpwiVi8n22K6QhrLBYdtApMoIrIZpqs/cvayPZydMCzEuzYeDumRDSksrcSRSyUAWKXcVjCJIiKbYSu9bNOQnmltMiJbsCO7AEIAA7t4wN9dc/snkNUxiSIim2BLvWzTfKz0c8UoraiWNRYiE9N8qBgO5dkMJlFEZBNsqZcd6uuKHr6uqDEK7DpVJGssRABQWWPAz6drK+nHhLG0ga1gEkVENsHWetnSXXoc0iMbkH6uGGVVBvi7qzGgC9ditRVMoohIdrbYyzYtAbMzuxAGI9dpJ3mZbrqIDmOVclvCJIqIZGfqZft1UuOuINvoZQ8N8UInjSOKy6qQeZHVy0k+QgjppotoG+lkUC0mUUQku1sFNv2gVNpGL9vJQYkxpurlXJCYZHS64DouXbsBtaMSUb185Q6H6mASRUSyEkLcWurFxnrZMWFMokh+pv8fI3v6wFnFKuW2hEkUEcnKlnvZY/p2hlIBZOeX4mJxudzhUAdluukimgsO2xwmUUQkK9NVHlvsZXu6qDC0e2318h3ZvBpFba+4rAqHcmvn5MWwSrnNYRJFRLKS7jqy0V529M1SB9s5pEcy2JldAKMA+gW6I8jTWe5w6FeYRBGRbNpDL9u0BM2+s1dRVlkjczTU0ZjuyrPV/x8dHZMoIpLNT6dsv5fds7Mbunm7oMpgxO4zrF5ObafaYMSu7Jv102ykCC2ZYxJFRLIxDZHZci9boVAgWrpLj9XLqe0cOF+M0soa+LiqMKirp9zhUAOYRBGRLNpTLzv25nyt1KxCGFm9nNqIaShvbJjt1E8jc0yiiEgWB3LaTy97eKg3XFUOKLpeiWOXdXKHQx2EqQhtrI13MjoyJlFEJAtTaYP20MtWOSoxqk9nABzSo7ZxtvA6zheVwclBgajeneUOhxrBJIqIZNHeetkxN4f0TEMsRNZkKrA5oocP3NSOMkdDjWESRURt7lw77GWP6dsZCgVwIk8Pra5C7nDIzqVk1V7xtOWbLohJFBHJIKUd9rJ93dQYHOwJ4NYXHJE16MqrcSCntn6ara0nSeaYRBFRm2uvvWzpLj1WLycr+ul0IQxGgd5+bujm4yJ3ONQEJlFE1KZ0N9pvL9tUL2r3mSLcqDLIHA3Zq9SbNy/E2OhSSHQLkygialM/nWq/veywgE4I8tCgssaIvWdZvZwsr8ZgxI52Uj+NbCSJWrZsGUJCQqDRaBAREYH9+/c32T45ORlhYWHQaDQYOHAgtmzZYrZfCIF58+YhMDAQzs7OiI2NxenTp83ahISEQKFQmD0WLlxo1ubo0aO47777oNFoEBwcjEWLFlnmBRN1YO25l61QKHiXHlnVodwS6G5Uw9PFCUNuzsEj2yV7ErVu3TokJiZi/vz5OHToEAYNGoS4uDgUFDT8C2rv3r0YP348pkyZgsOHDyMhIQEJCQk4fvy41GbRokX49NNPsWLFCqSnp8PV1RVxcXGoqDC/o+add97BlStXpMcLL7wg7dPr9Rg3bhy6d++OjIwMLF68GAsWLMDKlSut80YQdQA1BiN2nmrfvezom3GnniyAEKxeTpZlmi84tq8fHB1k/4qm25D9E/roo4/wzDPPYPLkyejfvz9WrFgBFxcXfPnllw22/+STT3D//ffjtddeQ79+/fDuu+/innvuweeffw6g9irU0qVLMWfOHPz+97/H3XffjX/+85/Iy8vDhg0bzI7VqVMnBAQESA9XV1dp3zfffIOqqip8+eWXuOuuu/D444/jxRdfxEcffWS194LI3h3KLUFJefvuZUf28IGzkwO0+gqcyNPLHQ7ZGdOdq9Ht7KaLjkrWJKqqqgoZGRmIjY2VtimVSsTGxiItLa3B56SlpZm1B4C4uDip/fnz56HVas3aeHh4ICIiot4xFy5cCB8fHwwZMgSLFy9GTU2N2XlGjRoFlUpldp7s7Gxcu3atwdgqKyuh1+vNHkR0iz30sjVODojq7QvgVsFQIku4cLUMZwquw1GpkCrkk22T9bdYUVERDAYD/P3N50b4+/tDq9U2+BytVttke9Oftzvmiy++iLVr12LHjh2YNm0a3n//fbz++uu3PU/dc/xaUlISPDw8pEdwcHCjr52oI0q1k162qTQD50WRJZmS8mEh3vBwdpI5GmqO9lHlzgoSExOlv999991QqVSYNm0akpKSoFarW3XM2bNnmx1Xr9czkSK6KfdqOU7bSS/blAQeuViCgtIK+HXSyBwR2QPTUF57nS/YEcl6JcrX1xcODg7Izzev/pufn4+AgIAGnxMQENBke9OfLTkmAERERKCmpgY5OTlNnqfuOX5NrVbD3d3d7EFEtUxDefbQy/Zz1+Durh4AgJ1ZhTJHQ/agtKIa6eevAmj/V2o7ElmTKJVKhfDwcKSkpEjbjEYjUlJSEBkZ2eBzIiMjzdoDwLZt26T2oaGhCAgIMGuj1+uRnp7e6DEBIDMzE0qlEn5+ftJ5du3aherqarPz9O3bF15eXi1/sUQdnGmowl562dHSkB6XgKE7t/t0EaoNAj18XdGjs5vc4VAzyT6zMzExEX//+9/x1Vdf4eTJk3juuedQVlaGyZMnAwAmTpyI2bNnS+1nzpyJrVu3YsmSJcjKysKCBQtw8OBBzJgxA0BtHZeXXnoJf/nLX7Bx40YcO3YMEydORFBQEBISEgDUThpfunQpjhw5gnPnzuGbb77Byy+/jCeeeEJKkP785z9DpVJhypQpOHHiBNatW4dPPvnEbLiOiJqntKIa+87ZVy875ma19Z9PF6GimtXL6c5st5P5gh2N7HOiHnvsMRQWFmLevHnQarUYPHgwtm7dKk3izs3NhVJ5K9cbOXIk1qxZgzlz5uDNN99E7969sWHDBgwYMEBq8/rrr6OsrAxTp05FSUkJoqKisHXrVmg0tfMW1Go11q5diwULFqCyshKhoaF4+eWXzRIkDw8P/Pjjj5g+fTrCw8Ph6+uLefPmYerUqW30zhDZD3vsZQ/o4g5/dzXy9ZVIP1+M0e18nhfJx2AU2JltulLb/orQdmQKwWpxVqPX6+Hh4QGdTsf5UdShvfLvI/jvoUt4OioUc37bX+5wLGb2+qP41/6LmBjZHe/8fsDtn0DUgIwL1/Dw8r3opHHEobm/gVM7Lf9hT5r7/c1Pioisqm4vO9pO5kOZmBZQTmH1croDqTfn1Y3u05kJVDvDT4uIrOrIpRJcLatCJ40jhoV4yx2ORUX18oXaUYnLJTdwKv+63OFQO2UqbRDLobx2h0kUEVlVykn77WU7qxwwsqcPAGD7Sd6lRy136Vo5srSlUCrAeXXtkH39RiMim2PvBQSjb1494BIw1Bo7bv67Ce/uBS9X1W1ak61hEkVEVnO55IbUyx7Txz6TKNMSMIdyr6G4rErmaKi9ScniXXntGZMoIrKa1JtDXPbcyw7ydEa/QHcIceuqAlFzlFfVYO/Z2vppMawP1S4xiSIiqzH1sk13sdkr0xcgh/SoJXafLkJVjRHB3s7o5Wcf9dM6GiZRRGQVdXvZsXY6H8rENN9r16lCVNUYZY6G2gtpKaQwfygUCpmjodZgEkVEVtGRetmDunrCx1WF0soaHMgpljscageMRlFnPpR9dzLsGZMoIrKKjtTLVioVGGtakPgkh/To9o7n6VBYWglXlQOGh9pX/bSOhEkUEVmc0ShuJVEdpJdtGrJMycpn9XK6LVOyPapPZ6gdHWSOhlqLSRQRWdzxPB0KOlgvO6p3Z6gclLhwtRxnC8vkDodsXMrNpV6ieVdeu8YkiogsriP2st3UjojoUZswmtZCI2qIVleB45f1UCggDQNT+8QkiogsLlUqbdCxviBiOC+KmmHHzQW5Bwd7wtdNLXM0dCeYRBGRReXrK3Dssq5D9rJNVacPXrgGXXm1zNGQrTKtJ8kCm+0fkygisijTVaiO2MsO9nZBH383GIwCO0/xahTVV1FtwO4zRQDsvwhtR8AkiogsSlpwuIP2sk1fjKxeTg1JO3sVFdVGBHlo0C+wk9zh0B1iEkVEFlPbyy4E0HF72aaSDjuzC1FjYPVyMrf95lBedD8/u6+f1hEwiSIii2EvG7inmxc8XZygu1GNjAvX5A6HbIgQwqwILbV/TKKIyGKk2jcduJftoFRgbF8uSEz1nbxSiiu6Cjg7OSCyp4/c4ZAFMIkiIosQQiD1JHvZwK3SDqahGyLg1l159/byhcapY9RPs3dMoojIIk5eKUUee9kAaouMOioVOFtYhpwiVi+nWlxw2P4wiSIiizBV6WYvG/BwdsKwEFP1cg7pEVBYWokjl0oAdLwitPaMSRQRWcT2k+xl1xVTZ0Fioh3ZBRACGNjFA/7uGrnDIQthEkVEd4y97PpM1cvTzxWjtILVyzu6VHYy7BKTKCK6YzvZy64n1NcVPXxdUWMU+Pl0kdzhkIwqawz4+XRt/bSOftOFvWESRUR3zFSlnFehzPEuPQJqr0aWVRng10mNu4Lc5Q6HLIhJFBHdkbq97Nh+7GXXZRrS25ldCINRyBwNySW1zl15SmXHrJ9mr5hEEdEdYS+7cUNDvNBJ44jisipkXmT18o5ICHFrqRcO5dkdJlFEdEdMvezoMPayf83JQYnRfToDuDXkSR3L6YLruHTtBlSOStzbq2PXT7NHTKKIqNWEENIt/DEcymuQaYiT9aI6JlPyfG9PH7ioHGWOhizNJpKoZcuWISQkBBqNBhEREdi/f3+T7ZOTkxEWFgaNRoOBAwdiy5YtZvuFEJg3bx4CAwPh7OyM2NhYnD59Wtqfk5ODKVOmIDQ0FM7OzujZsyfmz5+PqqoqszYKhaLeY9++fZZ98UTt2OmC67hYzF52U0b36QylAsjSluLStXK5w6E2ZlrqJZqdDLskexK1bt06JCYmYv78+Th06BAGDRqEuLg4FBQ03Gvbu3cvxo8fjylTpuDw4cNISEhAQkICjh8/LrVZtGgRPv30U6xYsQLp6elwdXVFXFwcKioqAABZWVkwGo3429/+hhMnTuDjjz/GihUr8Oabb9Y73/bt23HlyhXpER4ebp03gqgdMvWyR7KX3SgvVxXCu3sB4NWojqa4rAqHcmvnwvHOVTslZDZ8+HAxffp06WeDwSCCgoJEUlJSg+0fffRRER8fb7YtIiJCTJs2TQghhNFoFAEBAWLx4sXS/pKSEqFWq8W//vWvRuNYtGiRCA0NlX4+f/68ACAOHz7cmpclhBBCp9MJAEKn07X6GES27JHle0T3WZvEP9Ny5A7Fpi3feUZ0n7VJTPwiXe5QqA2tP3RRdJ+1Sdy/dJfcoVALNff7W9YrUVVVVcjIyEBsbKy0TalUIjY2FmlpaQ0+Jy0tzaw9AMTFxUntz58/D61Wa9bGw8MDERERjR4TAHQ6Hby9vettf+ihh+Dn54eoqChs3LixyddTWVkJvV5v9iCyV9fKqpBxgb3s5oi5+f6knb2KssoamaOhtiIthcT/H3ZL1iSqqKgIBoMB/v7mY8X+/v7QarUNPker1TbZ3vRnS4555swZfPbZZ5g2bZq0zc3NDUuWLEFycjI2b96MqKgoJCQkNJlIJSUlwcPDQ3oEBwc32paovdt5qgBGAYQFdEIXT2e5w7FpvfzcEOztjCqDEbvPsHp5R1BtMGJXdm39tGgu9WK3ZJ8TJbfLly/j/vvvx5/+9Cc888wz0nZfX18kJiYiIiICw4YNw8KFC/HEE09g8eLFjR5r9uzZ0Ol00uPixYtt8RKIZGGaD8UCm7enUCik5T5SWeqgQziQU4zSyhr4uKowuKun3OGQlciaRPn6+sLBwQH5+eZLIuTn5yMgIKDB5wQEBDTZ3vRnc46Zl5eHsWPHYuTIkVi5cuVt442IiMCZM2ca3a9Wq+Hu7m72ILJH1QYjfjrFXnZLmBaeTckqgJHVy+2eqZMxlvXT7JqsSZRKpUJ4eDhSUlKkbUajESkpKYiMjGzwOZGRkWbtAWDbtm1S+9DQUAQEBJi10ev1SE9PNzvm5cuXMWbMGISHh2PVqlVQKm//VmRmZiIwMLBFr5HIHh3IKUZpRW0vexB72c0yPNQbrioHFF2vxLHLOrnDISuTlnrhfCi7Jvs9yYmJiZg0aRKGDh2K4cOHY+nSpSgrK8PkyZMBABMnTkSXLl2QlJQEAJg5cyZGjx6NJUuWID4+HmvXrsXBgwelK0kKhQIvvfQS/vKXv6B3794IDQ3F3LlzERQUhISEBAC3Eqju3bvjww8/RGFhoRSP6WrVV199BZVKhSFDhgAA1q9fjy+//BL/+Mc/2uqtIbJZqXV62Q7sZTeL2tEBo/p0xg/HtUjJKsCgYE+5QyIrOVd4HeeLyuDkoMB9NyvWk32SPYl67LHHUFhYiHnz5kGr1WLw4MHYunWrNDE8NzfX7CrRyJEjsWbNGsyZMwdvvvkmevfujQ0bNmDAgAFSm9dffx1lZWWYOnUqSkpKEBUVha1bt0Kj0QCovXJ15swZnDlzBl27djWLR4hbl9nfffddXLhwAY6OjggLC8O6devwyCOPWPPtIGoXUtjLbpXoML/aJOpkPhJ/00fucMhKTEN5I3r4wE0t+9csWZFC1M0ayKL0ej08PDyg0+k4P4rsxrnC64he8hOcHBQ4NPc36KRxkjukdqPoeiWGvbcdQgD7ZscgwEMjd0hkBY+vTMO+c8WY/7v+mHxvqNzhUCs09/u7w9+dR0QtY5rrMaKHDxOoFvJ1U2PwzWE8Vi+3T7ob1TiQU1s/zXRHJtkvJlFE1CLbTWuBcSivVUxDoKY11ci+/HSqEAajQG8/N3TzcZE7HLIyJlFE1GzsZd+56Jvv2+4zRbhRZZA5GrK0VGnBYXYyOgImUUTUbLvYy75j/QI7IchDg8oaI9LOsXq5PakxGLHzZv00FqHtGJhEEVGzpbCXfccUCoX0/m1n9XK7cii3BCXl1fB0ccIQlrDoEJhEEVGzsJdtOTH9bi0Bwxuk7UdKVm0nY0yfznB04NdrR8BPmYia5fBF9rItJbKHD5ydHKDVV+CXK3q5wyELMRWhjWEno8NgEkVEzWK6K4+97DuncXLAvb18AdwqzEjtW+7VcpwuuA4HpQKjWKW8w+BvQiJqFvayLSu2zoLE1P6ZhvKGhXjBw5n10zoKJlFEdFvsZVve2Jv1oo5cLEFhaaXM0dCdMhVP5XzBjoVJFBHdFnvZlufvrsHALh4AgB28GtWulVZUY9+5qwBYhLajYRJFRLfFXrZ1xEhDeqxe3p7tPl2EaoNAD19X9OjsJnc41IaYRBFRk65X1rCXbSWmqu8/ny5CZQ2rl7dXpnlt/P/R8TCJIqIm/XyqENUGgVD2si3uriB3+HVSo7zKgH3niuUOh1rBYBTScCyL0HY8TKKIqEmmXnYMe9kWp1QqpCG9VC5I3C4duVSCq2VV6KRxxLAQb7nDoTbGJIqIGsVetvWZFiTezurl7ZJpKaTRfTrDifXTOhx+4kTUKPayre/eXj5QOSpxueQGTuVflzscaqEUqX4aOxkdEZMoImqUqcAme9nW46JyxL09fQDwLr325nLJDWRpS6FUAGP6MInqiPhbkYgaZVrqhb1s64q+WTqCS8C0L6Z5bOHdveDlqpI5GpIDkygiahB72W3HdGv8odxrKC6rkjkaaq5bpQ1YP62jYhJFRA0yFdhkL9v6ung6o1+gO4QAdmbzalR7UF5Vg71na+unxfJKbYfFJIqIGmS664i97LZhKiHBIb32YffpIlTVGBHs7Yxefqyf1lExiSKieur2sjkfqm2YSkjsOlWIqhqjzNHQ7aRK9dP8oVAoZI6G5MIkiojq2XPmqtTL7s1edpsY3NUTPq4qlFbW4GAOq5fbMqNR3Eqi2Mno0JhEEVE9pqE89rLbjlKpwNibQ3rbOaRn047n6VBQWglXlQOGh7J+WkfGJIqIzNTtZXNB1bYlzYvKymf1chtmmrd2X+/OUDs6yBwNyYlJFBGZOZGnl3rZET3Yy25L9/XpDCcHBS5cLce5ojK5w6FGcCiPTJhEEZEZU4FN9rLbnpvaESN63KxezgWJbVK+vgLHLuugUABj+jKJ6uiYRBGRmVQuOCyraJY6sGmm/x+Dunqicye1zNGQ3JhEEZGkbi97LHvZsoi5WZfr4IVr0JVXyxwN/ZopuWWBTQKYRBFRHexly6+bjwt6+7nBYBTYeYpXo2xJRbUBu88UAmARWqplE0nUsmXLEBISAo1Gg4iICOzfv7/J9snJyQgLC4NGo8HAgQOxZcsWs/1CCMybNw+BgYFwdnZGbGwsTp8+bdamuLgYEyZMgLu7Ozw9PTFlyhRcv37drM3Ro0dx3333QaPRIDg4GIsWLbLMCyayUaZedgzvypOVaSjVlNSSbUg7exUV1UYEeWjQL7CT3OGQDZA9iVq3bh0SExMxf/58HDp0CIMGDUJcXBwKChr+5bF3716MHz8eU6ZMweHDh5GQkICEhAQcP35carNo0SJ8+umnWLFiBdLT0+Hq6oq4uDhUVFRIbSZMmIATJ05g27Zt2LRpE3bt2oWpU6dK+/V6PcaNG4fu3bsjIyMDixcvxoIFC7By5UrrvRlEMqqoNmDPmSIAQEw/9rLlFHvz/d+ZXYgaA6uX24qUrJtLIfXzY/00AgAohMzFSCIiIjBs2DB8/vnnAACj0Yjg4GC88MILeOONN+q1f+yxx1BWVoZNmzZJ20aMGIHBgwdjxYoVEEIgKCgIr7zyCl599VUAgE6ng7+/P1avXo3HH38cJ0+eRP/+/XHgwAEMHToUALB161Y8+OCDuHTpEoKCgrB8+XK89dZb0Gq1UKlqF1994403sGHDBmRlZTXrten1enh4eECn08Hd3f2O3qe6Ll0rt9ixiEwO5BTj5XVHEOihwd43ovklIaMagxFD39uOkvJqrHjiHgzo4iF3SATg0RVpyNNVYNVTw6TCqGSfmvv97diGMdVTVVWFjIwMzJ49W9qmVCoRGxuLtLS0Bp+TlpaGxMREs21xcXHYsGEDAOD8+fPQarWIjY2V9nt4eCAiIgJpaWl4/PHHkZaWBk9PTymBAoDY2FgolUqkp6fjD3/4A9LS0jBq1CgpgTKd54MPPsC1a9fg5eVVL7bKykpUVlZKP+v1+pa9Ic0UveQnrq1FVhMdxl623BwdlBjTpzM2ZObh2a8PyR0O1aFxUiKyp4/cYZCNkDWJKioqgsFggL+/+dCBv79/o1d7tFptg+21Wq2037StqTZ+fua9CEdHR3h7e5u1CQ0NrXcM076GkqikpCS8/fbbjb9gC1E7KsGvOLIGd2cnjB/eTe4wCMCTkd2x+0wRSitq5A6FblIqFJh8bwg0TqyfRrVkTaLszezZs82ukun1egQHB1v8PMcWxFn8mERkW8K7e+PgnN/IHQYRNUHWieW+vr5wcHBAfr55Zd78/HwEBAQ0+JyAgIAm25v+vF2bX09cr6mpQXFxsVmbho5R9xy/plar4e7ubvYgIiIi+yRrEqVSqRAeHo6UlBRpm9FoREpKCiIjIxt8TmRkpFl7ANi2bZvUPjQ0FAEBAWZt9Ho90tPTpTaRkZEoKSlBRkaG1CY1NRVGoxERERFSm127dqG6utrsPH379m1wKI+IiIg6GCGztWvXCrVaLVavXi1++eUXMXXqVOHp6Sm0Wq0QQognn3xSvPHGG1L7PXv2CEdHR/Hhhx+KkydPivnz5wsnJydx7Ngxqc3ChQuFp6en+O6778TRo0fF73//exEaGipu3Lghtbn//vvFkCFDRHp6uti9e7fo3bu3GD9+vLS/pKRE+Pv7iyeffFIcP35crF27Vri4uIi//e1vzX5tOp1OABA6ne5O3iIiIiJqQ839/pY9iRJCiM8++0x069ZNqFQqMXz4cLFv3z5p3+jRo8WkSZPM2v/73/8Wffr0ESqVStx1111i8+bNZvuNRqOYO3eu8Pf3F2q1WsTExIjs7GyzNlevXhXjx48Xbm5uwt3dXUyePFmUlpaatTly5IiIiooSarVadOnSRSxcuLBFr4tJFBERUfvT3O9v2etE2TNr1YkiIiIi62nu97fsFcuJiIiI2iMmUUREREStwCSKiIiIqBWYRBERERG1ApMoIiIiolZgEkVERETUCkyiiIiIiFqBSRQRERFRKzCJIiIiImoFR7kDsGemYvB6vV7mSIiIiKi5TN/bt1vUhUmUFZWWlgIAgoODZY6EiIiIWqq0tBQeHh6N7ufaeVZkNBqRl5eHTp06QaFQWOy4er0ewcHBuHjxItfkswH8PGwLPw/bw8/EtvDzuD0hBEpLSxEUFASlsvGZT7wSZUVKpRJdu3a12vHd3d35H8CG8POwLfw8bA8/E9vCz6NpTV2BMuHEciIiIqJWYBJFRERE1ApMotohtVqN+fPnQ61Wyx0KgZ+HreHnYXv4mdgWfh6Ww4nlRERERK3AK1FERERErcAkioiIiKgVmEQRERERtQKTKCIiIqJWYBLVDi1btgwhISHQaDSIiIjA/v375Q7J7iQlJWHYsGHo1KkT/Pz8kJCQgOzsbLM2FRUVmD59Onx8fODm5oaHH34Y+fn5Zm1yc3MRHx8PFxcX+Pn54bXXXkNNTU1bvhS7tHDhQigUCrz00kvSNn4ebevy5ct44okn4OPjA2dnZwwcOBAHDx6U9gshMG/ePAQGBsLZ2RmxsbE4ffq02TGKi4sxYcIEuLu7w9PTE1OmTMH169fb+qXYBYPBgLlz5yI0NBTOzs7o2bMn3n33XbO13/iZWIGgdmXt2rVCpVKJL7/8Upw4cUI888wzwtPTU+Tn58sdml2Ji4sTq1atEsePHxeZmZniwQcfFN26dRPXr1+X2jz77LMiODhYpKSkiIMHD4oRI0aIkSNHSvtramrEgAEDRGxsrDh8+LDYsmWL8PX1FbNnz5bjJdmN/fv3i5CQEHH33XeLmTNnStv5ebSd4uJi0b17d/HUU0+J9PR0ce7cOfG///1PnDlzRmqzcOFC4eHhITZs2CCOHDkiHnroIREaGipu3Lghtbn//vvFoEGDxL59+8TPP/8sevXqJcaPHy/HS2r33nvvPeHj4yM2bdokzp8/L5KTk4Wbm5v45JNPpDb8TCyPSVQ7M3z4cDF9+nTpZ4PBIIKCgkRSUpKMUdm/goICAUD89NNPQgghSkpKhJOTk0hOTpbanDx5UgAQaWlpQgghtmzZIpRKpdBqtVKb5cuXC3d3d1FZWdm2L8BOlJaWit69e4tt27aJ0aNHS0kUP4+2NWvWLBEVFdXofqPRKAICAsTixYulbSUlJUKtVot//etfQgghfvnlFwFAHDhwQGrzww8/CIVCIS5fvmy94O1UfHy8+L//+z+zbX/84x/FhAkThBD8TKyFw3ntSFVVFTIyMhAbGyttUyqViI2NRVpamoyR2T+dTgcA8Pb2BgBkZGSgurra7LMICwtDt27dpM8iLS0NAwcOhL+/v9QmLi4Oer0eJ06caMPo7cf06dMRHx9v9r4D/Dza2saNGzF06FD86U9/gp+fH4YMGYK///3v0v7z589Dq9WafR4eHh6IiIgw+zw8PT0xdOhQqU1sbCyUSiXS09Pb7sXYiZEjRyIlJQWnTp0CABw5cgS7d+/GAw88AICfibVwAeJ2pKioCAaDwexLAAD8/f2RlZUlU1T2z2g04qWXXsK9996LAQMGAAC0Wi1UKhU8PT3N2vr7+0Or1UptGvqsTPuoZdauXYtDhw7hwIED9fbx82hb586dw/Lly5GYmIg333wTBw4cwIsvvgiVSoVJkyZJ72dD73fdz8PPz89sv6OjI7y9vfl5tMIbb7wBvV6PsLAwODg4wGAw4L333sOECRMAgJ+JlTCJIrqN6dOn4/jx49i9e7fcoXRYFy9exMyZM7Ft2zZoNBq5w+nwjEYjhg4divfffx8AMGTIEBw/fhwrVqzApEmTZI6uY/r3v/+Nb775BmvWrMFdd92FzMxMvPTSSwgKCuJnYkUczmtHfH194eDgUO+Oo/z8fAQEBMgUlX2bMWMGNm3ahB07dqBr167S9oCAAFRVVaGkpMSsfd3PIiAgoMHPyrSPmi8jIwMFBQW455574OjoCEdHR/z000/49NNP4ejoCH9/f34ebSgwMBD9+/c329avXz/k5uYCuPV+NvW7KiAgAAUFBWb7a2pqUFxczM+jFV577TW88cYbePzxxzFw4EA8+eSTePnll5GUlASAn4m1MIlqR1QqFcLDw5GSkiJtMxqNSElJQWRkpIyR2R8hBGbMmIFvv/0WqampCA0NNdsfHh4OJycns88iOzsbubm50mcRGRmJY8eOmf1S2rZtG9zd3et9AVHTYmJicOzYMWRmZkqPoUOHYsKECdLf+Xm0nXvvvbdeyY9Tp06he/fuAIDQ0FAEBASYfR56vR7p6elmn0dJSQkyMjKkNqmpqTAajYiIiGiDV2FfysvLoVSaf6U7ODjAaDQC4GdiNXLPbKeWWbt2rVCr1WL16tXil19+EVOnThWenp5mdxzRnXvuueeEh4eH2Llzp7hy5Yr0KC8vl9o8++yzolu3biI1NVUcPHhQREZGisjISGm/6Zb6cePGiczMTLF161bRuXNn3lJvIXXvzhOCn0db2r9/v3B0dBTvvfeeOH36tPjmm2+Ei4uL+Prrr6U2CxcuFJ6enuK7774TR48eFb///e8bvJ1+yJAhIj09XezevVv07t2bt9O30qRJk0SXLl2kEgfr168Xvr6+4vXXX5fa8DOxPCZR7dBnn30munXrJlQqlRg+fLjYt2+f3CHZHQANPlatWiW1uXHjhnj++eeFl5eXcHFxEX/4wx/ElStXzI6Tk5MjHnjgAeHs7Cx8fX3FK6+8Iqqrq9v41dinXydR/Dza1vfffy8GDBgg1Gq1CAsLEytXrjTbbzQaxdy5c4W/v79Qq9UiJiZGZGdnm7W5evWqGD9+vHBzcxPu7u5i8uTJorS0tC1fht3Q6/Vi5syZolu3bkKj0YgePXqIt956y6x8Bz8Ty1MIUaecKRERERE1C+dEEREREbUCkygiIiKiVmASRURERNQKTKKIiIiIWoFJFBEREVErMIkiIiIiagUmUUREREStwCSKiDq8nJwcKBQKZGZmWu0cTz31FBISEqx2fCJqe0yiiKjde+qpp6BQKOo97r///mY9Pzg4GFeuXMGAAQOsHCkR2RNHuQMgIrKE+++/H6tWrTLbplarm/VcBwcHrlJPRC3GK1FEZBfUajUCAgLMHl5eXgAAhUKB5cuX44EHHoCzszN69OiB//znP9Jzfz2cd+3aNUyYMAGdO3eGs7MzevfubZagHTt2DNHR0XB2doaPjw+mTp2K69evS/sNBgMSExPh6ekJHx8fvP766/j1CltGoxFJSUkIDQ2Fs7MzBg0aZBbT7WIgIvkxiSKiDmHu3Ll4+OGHceTIEUyYMAGPP/44Tp482WjbX375BT/88ANOnjyJ5cuXw9fXFwBQVlaGuLg4eHl54cCBA0hOTsb27dsxY8YM6flLlizB6tWr8eWXX2L37t0oLi7Gt99+a3aOpKQk/POf/8SKFStw4sQJvPzyy3jiiSfw008/3TYGIrIRMi+ATER0xyZNmiQcHByEq6ur2eO9994TQggBQDz77LNmz4mIiBDPPfecEEKI8+fPCwDi8OHDQgghfve734nJkyc3eK6VK1cKLy8vcf36dWnb5s2bhVKpFFqtVgghRGBgoFi0aJG0v7q6WnTt2lX8/ve/F0IIUVFRIVxcXMTevXvNjj1lyhQxfvz428ZARLaBc6KIyC6MHTsWy5cvN9vm7e0t/T0yMtJsX2RkZKN34z333HN4+OGHcejQIYwbNw4JCQkYOXIkAODkyZMYNGgQXF1dpfb33nsvjEYjsrOzodFocOXKFUREREj7HR0dMXToUGlI78yZMygvL8dvfvMbs/NWVVVhyJAht42BiGwDkygisguurq7o1auXRY71wAMP4MKFC9iyZQu2bduGmJgYTJ8+HR9++KFFjm+aP7V582Z06dLFbJ9pMry1YyCiO8c5UUTUIezbt6/ez/369Wu0fefOnTFp0iR8/fXXWLp0KVauXAkA6NevH44cOYKysjKp7Z49e6BUKtG3b194eHggMDAQ6enp0v6amhpkZGRIP/fv3x9qtRq5ubno1auX2SM4OPi2MRCRbeCVKCKyC5WVldBqtWbbHB0dpcnYycnJGDp0KKKiovDNN99g//79+OKLLxo81rx58xAeHo677roLlZWV2LRpk5RwTZgwAfPnz8ekSZOwYMECFBYW4oUXXsCTTz4Jf39/AMDMmTOxcOFC9O7dG2FhYfjoo49QUlIiHb9Tp0549dVX8fLLL8NoNCIqKgo6nQ579uyBu7s7Jk2a1GQMRGQbmEQRkV3YunUrAgMDzbb17dsXWVlZAIC3334ba9euxfPPP4/AwED861//Qv/+/Rs8lkqlwuzZs5GTkwNnZ2fcd999WLt2LQDAxcUF//vf/zBz5kwMGzYMLi4uePjhh/HRRx9Jz3/llVdw5coVTJo0CUqlEv/3f/+HP/zhD9DpdFKbd999F507d0ZSUhLOnTsHT09P3HPPPXjzzTdvGwMR2QaFEL8qXkJEZGcUCgW+/fZbLrtCRBbFOVFERERErcAkioiIiKgVOCeKiOweZy0QkTXwShQRERFRKzCJIiIiImoFJlFERERErcAkioiIiKgVmEQRERERtQKTKCIiIqJWYBJFRERE1ApMooiIiIhagUkUERERUSv8f2zzwNlzma3dAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "win_array = np.array(win_list)\n",
    "#每100条轨迹取一次平均\n",
    "win_array = np.mean(win_array.reshape(-1, 100), axis=1)\n",
    "\n",
    "episodes_list = np.arange(win_array.shape[0]) * 100\n",
    "plt.plot(episodes_list, win_array)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Win rate')\n",
    "plt.title('IPPO on Combat')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c6a8fd92",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from gym import spaces\n",
    "import numpy as np\n",
    "from ma_gym.envs.utils.action_space import MultiAgentActionSpace\n",
    "\n",
    "class ActionSpaceWrapper(gym.Wrapper):\n",
    "    def __init__(self, env):\n",
    "        super(ActionSpaceWrapper, self).__init__(env)\n",
    "        # 假设每个智能体有相同的动作空间\n",
    "        self.n_agents = env.n_agents\n",
    "        self.single_action_space = env.action_space[0]\n",
    "        # 创建一个 Tuple 动作空间\n",
    "        self.action_space = spaces.Tuple([self.single_action_space for _ in range(self.n_agents)])\n",
    "    \n",
    "    def step(self, actions):\n",
    "        # 将 Tuple 动作转换为 ma-gym 期望的格式\n",
    "        return self.env.step(actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "72fa2595",
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'pyglet_rendering' from 'gym.utils' (/mnt/d/Code/Python/RL-Learning/.venv/lib/python3.11/site-packages/gym/utils/__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mImportError\u001b[39m                               Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[12]\u001b[39m\u001b[32m, line 11\u001b[39m\n\u001b[32m      9\u001b[39m obs_n = env.reset()\n\u001b[32m     10\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mall\u001b[39m(done_n):\n\u001b[32m---> \u001b[39m\u001b[32m11\u001b[39m     \u001b[43menv\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     12\u001b[39m     \u001b[38;5;66;03m# a_1 = agent.take_action(obs_n[0])\u001b[39;00m\n\u001b[32m     13\u001b[39m     \u001b[38;5;66;03m# a_2 = agent.take_action(obs_n[1])\u001b[39;00m\n\u001b[32m     14\u001b[39m     obs_n, reward_n, done_n, info = env.step(env.action_space.sample())\n",
      "\u001b[36mFile \u001b[39m\u001b[32m/mnt/d/Code/Python/RL-Learning/强化学习前沿篇/../ma_gym/envs/combat/combat.py:238\u001b[39m, in \u001b[36mCombat.render\u001b[39m\u001b[34m(self, mode)\u001b[39m\n\u001b[32m    234\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m img\n\u001b[32m    235\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m mode == \u001b[33m'\u001b[39m\u001b[33mhuman\u001b[39m\u001b[33m'\u001b[39m:\n\u001b[32m    236\u001b[39m     \u001b[38;5;66;03m# from gym.envs.classic_control import rendering\u001b[39;00m\n\u001b[32m    237\u001b[39m     \u001b[38;5;66;03m# from gymnasium.envs.classic_control import rendering\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m238\u001b[39m     \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mgym\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m pyglet_rendering \u001b[38;5;28;01mas\u001b[39;00m rendering\n\u001b[32m    239\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.viewer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    240\u001b[39m         \u001b[38;5;28mself\u001b[39m.viewer = rendering.SimpleImageViewer()\n",
      "\u001b[31mImportError\u001b[39m: cannot import name 'pyglet_rendering' from 'gym.utils' (/mnt/d/Code/Python/RL-Learning/.venv/lib/python3.11/site-packages/gym/utils/__init__.py)"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import time\n",
    "\n",
    "# env = gym.make('ma_gym:combat-v0', disable_env_checker=True)\n",
    "# env = ActionSpaceWrapper(env)\n",
    "done_n = [False for _ in range(env.n_agents)]\n",
    "ep_reward = 0\n",
    "\n",
    "obs_n = env.reset()\n",
    "while not all(done_n):\n",
    "    env.render()\n",
    "    # a_1 = agent.take_action(obs_n[0])\n",
    "    # a_2 = agent.take_action(obs_n[1])\n",
    "    obs_n, reward_n, done_n, info = env.step(env.action_space.sample())\n",
    "    # obs_n, reward_n, done_n, info = env.step([a_1, a_2])\n",
    "    ep_reward += sum(reward_n)\n",
    "    time.sleep(0.1)\n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
